# -*- coding: utf-8 -*-
import scrapy
from dingxiangyuan.items import DingxiangyuanItemLoader, PostsRepliesItem
from dingxiangyuan.DBHelper import db_conn, redis_conn
import pandas as pd
from dingxiangyuan.settings import BOARD_ID_MAP


def get_topic_left_start_urls():
    topic_urls_data = pd.read_sql(sql="select distinct topic_url from posts_replies", con=db_conn)
    topic_urls_floor_one = pd.read_sql(sql="select topic_url from posts_replies where floor=1", con=db_conn)
    has_topic_urls = set(topic_urls_data['topic_url']) - set(topic_urls_floor_one['topic_url'])
    topic_page_urls = redis_conn.smembers('topic_page_urls')
    start_urls = {url.decode() for url in topic_page_urls if url.decode().split('?')[0] in has_topic_urls}
    # print(start_urls, len(start_urls))
    return start_urls


class PostsRepliesSpider(scrapy.Spider):
    name = 'posts_replies'
    allowed_domains = ['www.dxy.cn', 'neuro.dxy.cn', 'chest.dxy.cn', 'cardiovascular.dxy.cn']
    start_urls = get_topic_left_start_urls()

    custom_settings = {
        'DEFAULT_REQUEST_HEADERS': {
            'Cookie': 'DXY_USER_GROUP=63; __auc=ca4a673d173746d32cbbeca9249; _ga=GA1.2.1989255604.1595386166; CLASS_CASTGC=146f361ccb637797b810e7e2328da5fff0605920d9642d4bbee447b01549249787dcf249fcba8e188be34b18d84f64a065e145c9dfaa7926fdc5c615144d617d7ba2e9371f7e928ee08ba85c3446c6a32bbda0e2f137a2f41e9cbe7ba30aefa7136219eee50aec04ab8c61eb1055ee470396f4486cc8f9e9ccc2e5af144ea0bf8ddba7280ba761f714eab85f13513c52848d2707b1be6e68a4a59b53ae8fa59c3ca524c67ac52ff7518e85d40a21df8d7b8ce33294c438e128026afa338ec24865c8afaa78d9f64d434ecbc4da0054b6fee44c76a243a122017ce91b0557b784d40be788bc15b361207b4fd6f69671a1cbee08f94ede72294f54e01c3b074e5a; JUTE_BBS_DATA=49798dd5282ede3bc43bd853b52aff9a1dc750801848826068ae67a5adecdd2a0dfb1ad527cf8331b805f0eeb3f7f62422b83b82525c1e3706bd63b1d9ef0f614072c37a3ef5d311926c003ba59fb5d5; ifVisitOldVerBBS=true; route=fe9340375d898dc58dc46edbbdbb48f8; route_bbs=d183b59af94599ac5496eae0a79bc70f; dxy_da_cookie-id=3cb8b04f3626982530333173874a15971595470988552; JUTE_SESSION_ID=ac5d913a-22c3-478f-8254-7cf64669699f; JUTE_TOKEN=6ce4333d-1e0a-48cd-af1f-8b26ed55d4e4; cms_token=f52fb742-cc35-4aec-b186-69f7f98a2fe4; Hm_lvt_8a6dad3652ee53a288a11ca184581908=1595414051,1595470987,1595501771,1595520015; __asc=63bc4c541737c676aca74452e80; __utma=1.1989255604.1595386166.1595520020.1595520020.1; __utmc=1; __utmz=1.1595520020.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); Hm_lpvt_8a6dad3652ee53a288a11ca184581908=1595520958; __utmb=1.33.9.1595520959282; CMSSESSIONID=BEC94C113534CD3D7BBC5866272269E9-n2; JUTE_SESSION=c60636c249b07a0800b0f8fd8f284a382683f74376827ded7e32a706967e09b74a41ebd2e52b5e934b256040bdd27899d572c01064e0b36a9dae061c53371eb2d58cfae5a80a7bdd',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36',
        }
    }

    @staticmethod
    def get_start_urls(board_id=None):
        topic_page_urls = redis_conn.smembers('topic_page_urls')
        if board_id:
            data = pd.read_sql(sql="select topic_url from topics where board_id = %s", params=(board_id,), con=db_conn)
            topic_urls = set(data['topic_url'])
            topic_url_list = {url.decode() for url in topic_page_urls if url.decode().split('?')[0] in topic_urls}
        else:
            topic_url_list = {url.decode() for url in topic_page_urls}
        has_seen_urls= redis_conn.scard('posts_replies:dupefilter')
        has_count = len(topic_page_urls) - has_seen_urls
        print(f'共 {len(topic_url_list)}\t已下载 {has_seen_urls}\t还剩 {has_count}')
        return topic_url_list


    # def start_requests(self):
    #     for url in self.get_start_urls():
    #         yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        topic_url = response.url.rsplit('?', maxsplit=1)[0]
        topic_type = response.xpath('//div[@id="location"]/a[3]/text()')
        topic_type = topic_type.extract_first() if topic_type else '其它'
        board_name = response.xpath('//div[@id="board"]/div/h1/a/text()').extract_first()
        topic_title = self.strings_prosses(
            response.xpath('//*[@id="postview"]/table/tbody/tr/th/h1/text()').extract_first())
        post_good_list = response.xpath('//div[@id="post_good"]/div[@class="good_post"]/table/tbody/tr')
        if post_good_list:
            for good_div in post_good_list:
                item_loader = DingxiangyuanItemLoader(item=PostsRepliesItem(), selector=good_div)
                item_loader.add_value('topic_url', topic_url)
                if good_div.xpath('td[@class="tbc"]/div[1]/div[1]/ul/div[@class="blt-icon lfloat"]'):
                    item_loader.add_value('hot', 1)
                else:
                    item_loader.add_value('hot', 0)
                floor = int(good_div.xpath(
                    'td[@class="tbc"]/div[1]/div[1]/ul/li/span[@class="layer"]/text()').extract_first().strip('楼'))
                item_loader.add_value('floor', floor)
                item_loader.add_value('topic_title', topic_title)
                if floor == 1:
                    item_loader.add_value('isTopic', 1)
                    item_loader.add_xpath('browser_num',
                                          'td[@class="tbc"]/div[@class="conbox"]//div[@class="post-info"]/span[last()-1]/text()')
                    item_loader.add_xpath('reply_num',
                                          'td[@class="tbc"]/div[@class="conbox"]//div[@class="post-info"]/span[last()]/text()')
                else:
                    item_loader.add_value('isTopic', 0)
                    item_loader.add_value('browser_num', 0)
                    item_loader.add_value('reply_num', 0)
                item_loader.add_xpath('content', 'string(td[@class="tbc"]/div[@class="conbox"]//td[@class="postbody"])')
                item_loader.add_value('topic_type', topic_type)  # 主题帖类型
                item_loader.add_value('board_name', board_name)  # 所属板块
                item_loader.add_xpath('post_time',
                                      'td[@class="tbc"]/div[@class="conbox"]//div[@class="post-info"]/span[1]/text()')
                item_loader.add_xpath('author_name', 'td[@class="tbs"]/div[@class="auth"]/a/text()')
                item_loader.add_xpath('author_url', 'td[@class="tbs"]/div[@class="auth"]/a/@href')
                if good_div.xpath('td[@class="tbs"]/div[3]//span[@class="identify-icon"]'):
                    item_loader.add_xpath('author_identify_depart',
                                          'td[@class="tbs"]/div[3]/div[contains(@class, "expert")]/a/text()')
                    item_loader.add_xpath('author_identify',
                                          'td[@class="tbs"]/div[3]/div[contains(@class, "expert")]/a/@title')
                else:
                    item_loader.add_value('author_identify_depart', '无')
                    item_loader.add_value('author_identify', '无')
                if good_div.xpath('td[@class="tbs"]/div[3]/div[@class="user-level-area"]/text()'):
                    item_loader.add_xpath('author_level',
                                          'td[@class="tbs"]/div[3]/div[@class="user-level-area"]/text()')
                elif good_div.xpath('td[@class="tbs"]/div[3]//span[@class="adm"]/text()'):
                    item_loader.add_xpath('author_level', 'td[@class="tbs"]/div[3]//span[@class="adm"]/text()')
                else:
                    item_loader.add_value('author_level', '其它')
                item_loader.add_xpath('author_scores',
                                      'td[@class="tbs"]/div[@class="user_atten"]/ul/li[1]/div/a/text()')
                item_loader.add_xpath('author_votes', 'td[@class="tbs"]/div[@class="user_atten"]/ul/li[2]/div/a/text()')
                item_loader.add_xpath('author_dingdang',
                                      'td[@class="tbs"]/div[@class="user_atten"]/ul/li[3]/div/a/text()')
                item_loader.add_xpath('vote_num',
                                      'td[@class="tbc"]/div[@class="conbox"]//li[@class="vote-btn"]/span/text()')
                item_loader.add_xpath('fav_num',
                                      'td[@class="tbc"]/div[@class="conbox"]//li[@class="fav-btn"]/span/text()')
                item_loader.add_xpath('reward_num',
                                      'td[@class="tbc"]/div[@class="conbox"]//li[@class="reward-btn"]/span/text()')
                if good_div.xpath('td[@class="tbc"]/div[@class="conbox"]//div[@class="quote"]'):
                    item_loader.add_value('reference_bool', 1)
                else:
                    item_loader.add_value('reference_bool', 0)
                item_loader.add_value('isGood', 1)
                posts_item = item_loader.load_item()
                yield posts_item

        tr_list = response.xpath('//div[starts-with(@id, "post_")]/table/tbody/tr')
        for tr in tr_list:
            item_loader = DingxiangyuanItemLoader(item=PostsRepliesItem(), selector=tr)
            item_loader.add_value('topic_url', topic_url)
            if tr.xpath('td[@class="tbc"]/div[1]/div[1]/ul/div[@class="blt-icon lfloat"]'):
                item_loader.add_value('hot', 1)
            else:
                item_loader.add_value('hot', 0)
            floor = int(
                tr.xpath('td[@class="tbc"]/div[1]/div[1]/ul//span[@class="layer"]/text()').extract_first().strip('楼'))
            item_loader.add_value('floor', floor)
            item_loader.add_value('topic_title', topic_title)
            if floor == 1:
                item_loader.add_value('isTopic', 1)
                item_loader.add_xpath('browser_num',
                                      'td[@class="tbc"]/div[@class="conbox"]//div[@class="post-info"]/span[last()-1]/text()')
                item_loader.add_xpath('reply_num',
                                      'td[@class="tbc"]/div[@class="conbox"]//div[@class="post-info"]/span[last()]/text()')
            else:
                item_loader.add_value('isTopic', 0)
                item_loader.add_value('browser_num', 0)
                item_loader.add_value('reply_num', 0)
            item_loader.add_xpath('content', 'string(td[@class="tbc"]/div[@class="conbox"]//td[@class="postbody"])')
            item_loader.add_value('topic_type', topic_type)  # 主题帖类型
            item_loader.add_value('board_name', board_name)  # 所属板块
            item_loader.add_xpath('post_time',
                                  'td[@class="tbc"]/div[@class="conbox"]//div[@class="post-info"]/span[1]/text()')
            item_loader.add_xpath('author_name', 'td[@class="tbs"]/div[@class="auth"]/a/text()')
            item_loader.add_xpath('author_url', 'td[@class="tbs"]/div[@class="auth"]/a/@href')
            if tr.xpath('td[@class="tbs"]/div[3]//span[@class="identify-icon"]'):
                item_loader.add_xpath('author_identify_depart',
                                      'td[@class="tbs"]/div[3]/div[contains(@class, "expert")]/a/text()')
                item_loader.add_xpath('author_identify',
                                      'td[@class="tbs"]/div[3]/div[contains(@class, "expert")]/a/@title')
            else:
                item_loader.add_value('author_identify_depart', '无')
                item_loader.add_value('author_identify', '无')
            if tr.xpath('td[@class="tbs"]/div[3]/div[@class="user-level-area"]/text()'):
                item_loader.add_xpath('author_level', 'td[@class="tbs"]/div[3]/div[@class="user-level-area"]/text()')
            elif tr.xpath('td[@class="tbs"]/div[3]//span[@class="adm"]/text()'):
                item_loader.add_xpath('author_level', 'td[@class="tbs"]/div[3]//span[@class="adm"]/text()')
            else:
                item_loader.add_value('author_level', '其它')
            item_loader.add_xpath('author_scores', 'td[@class="tbs"]/div[@class="user_atten"]/ul/li[1]/div/a/text()')
            item_loader.add_xpath('author_votes', 'td[@class="tbs"]/div[@class="user_atten"]/ul/li[2]/div/a/text()')
            item_loader.add_xpath('author_dingdang', 'td[@class="tbs"]/div[@class="user_atten"]/ul/li[3]/div/a/text()')
            item_loader.add_xpath('vote_num',
                                  'td[@class="tbc"]/div[@class="conbox"]//li[@class="vote-btn"]/span/text()')
            item_loader.add_xpath('fav_num', 'td[@class="tbc"]/div[@class="conbox"]//li[@class="fav-btn"]/span/text()')
            item_loader.add_xpath('reward_num',
                                  'td[@class="tbc"]/div[@class="conbox"]//li[@class="reward-btn"]/span/text()')
            if tr.xpath('td[@class="tbc"]/div[@class="conbox"]//td[@class="postbody"]/div[@class="quote"]'):
                item_loader.add_value('reference_bool', 1)
            else:
                item_loader.add_value('reference_bool', 0)
            item_loader.add_value('isGood', 0)
            posts_item = item_loader.load_item()
            yield posts_item

    @staticmethod
    def strings_prosses(content):
        return content.strip().replace(' ', '')
